# -*- coding: utf-8 -*-

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from lxml import etree
import time
from fake_useragent import UserAgent
from pymongo import MongoClient

"""
远程调试
cmd 运行：
    chrome.exe --remote-debugging-port=9222 --user-data-dir="D:\webdriver\AutomationProfile"
添加参数
chrome_options.add_experimental_option("debuggerAddress", "127.0.0.1:9222")
"""

"""
驱动地址：https://sites.google.com/a/chromium.org/chromedriver/downloads
        http://npm.taobao.org/mirrors/chromedriver/
WebDriver 只能在一个页面上对元素识别与定位，对于frame/iframe 表单内嵌页面上的元素无法直接定位
需要通过switch_to.frame()方法将当前定位的主体切换为frame/iframe 表单的内嵌页面中
#切换到表单
driver.switch_to.frame("x-URS-iframe")
打开新的标签页
js=f'window.open("{url}");'
driver.execute_script(js)
------Select------
# 选中这个标签，然后使用Select创建对象
selectTag = Select(driver.find_element_by_name("jumpMenu"))
# 根据索引选择
selectTag.select_by_index(1)
# 根据值选择
selectTag.select_by_value("值")
# 根据可视的文本选择
selectTag.select_by_visible_text("显示值")
# 取消选中所有选项
selectTag.deselect_all()
------Select------
------切换窗口并关闭------
# 获取当前窗口句柄
now_handle = driver.current_window_handle
# 获取所有窗口句柄
all_handles = driver.window_handles
for handle in all_handles:
    if handle != now_handle:
        driver.switch_to.window(handle)  # 切换新打开窗口
        driver.close()
# 返回主窗口
driver.switch_to.window(now_handle)
------切换窗口并关闭------
------滚动条scrollTop=0为顶部，100000为底部------
js="var q=document.documentElement.scrollTop=100000"
------滚动条------
------返回521反爬-----
driver.get 两次.第一次获取加密cookie。第二次提交解析后的cookie
------返回521反爬-----
---------CSS 有空格------------
driver.find_element_by_css_selector('[class="btn btn-primary"]').click()
---------CSS 有空格------------
"""


# 初始化
def init_driver():
    chrome_options = Options()
    # 设置chrome不加载图片，提高速度
    # chrome_options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})
    # chrome_options.add_argument('--headless') # 无头模式
    chrome_options.add_argument('--disable-gpu')
    chrome_options.add_argument('user-agent='+UserAgent().random)  # 修改请求头，伪装浏览器型号
    chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])  # 设置开发模式，避免被检测
    # driver = webdriver.Chrome(chrome_options=chrome_options) # 已经设置PATH环境变量
    # 未在PATH环境变量中设置，需要设置驱动可执行路径
    driver = webdriver.Chrome(executable_path='D:\\webdriver\\chromedriver.exe', chrome_options=chrome_options)
    driver.maximize_window()  # 最大化

    # html = driver.page_source
    # print(html)

    return driver


# 获取页面url
def get_url(drive, url, country):
    driver.get(url)
    # 设置等待
    WebDriverWait(driver, 50).until(
        EC.presence_of_element_located((By.XPATH, '//a[contains(text(),"1")]')))
    js = 'var action=document.documentElement.scrollTop=7000'
    driver.execute_script(js)
    html = etree.HTML(driver.page_source)
    # xpath = html.xpath('//div[@class="list_product_right"]')
    xpath = html.xpath('//div[@class="list_product_box js_product_item"]')
    for x in xpath:
        supplier = "".join(x.xpath('./div/div/div[2]/div/div/p//text()'))  # 零售商
        url_id = x.xpath('./@data-track-product-id')[0]
        title = "".join(x.xpath('./div/div/div[2]/p/@title'))  # 标题
        pf = "".join(x.xpath('./div/div/div[2]/div/div[1]/div[1]/div/p//text()')).replace("分", "")  # 总评分
        renshu = "".join(x.xpath('./div/div/div[2]/div/div[1]/div[1]/div[2]/div[1]//text()'))  # 出行人数
        pingjia = "".join(x.xpath('./div/div/div[2]/div/div[1]/div[1]/div[2]/div[2]//text()'))  # 评价数量
        price = "".join(x.xpath('./div/div/div[2]/div/div[1]/div[2]/span//text()'))  # 价格
        # 评分<5
        if pf != "":
            if float(pf) < 6:
                url = f"https://vacations.ctrip.com/tour/booking/comment-detail/{url_id}"
                js=f'window.open("{url}");'
                driver.execute_script(js)
                time.sleep(1)
                # 获取当前窗口句柄
                now_handle = driver.current_window_handle
                # 获取所有窗口句柄
                all_handles = driver.window_handles
                for handle in all_handles:
                    if handle != now_handle:
                        driver.switch_to.window(handle)  # 切换新打开窗口
                        scroll_flag = True
                        while True:
                            # page_height = driver.execute_script('return document.documentElement.scrollHeight')  # 页面高度
                            if scroll_flag:
                                js = 'var action=document.documentElement.scrollTop=18800'
                                driver.execute_script(js)
                                scroll_flag = False
                            # height = page_height - 900
                            # js = f'var action=document.documentElement.scrollTop=10000'
                            # driver.execute_script(js)
                            time.sleep(0.5)
                            WebDriverWait(driver, 50).until(
                                EC.presence_of_element_located((By.XPATH, '//a[contains(text(),"1")]')))
                            # time.sleep(1)
                            html_child = etree.HTML(driver.page_source)
                            # 评价标签
                            # tag = html_child.xpath('//ul[@class="choice_box basefix"]/li')
                            # num = 0
                            # for t in tag:
                            #     tag_name = t.xpath('./label//text()')
                            #     num += 1
                            # 评论内容
                            detail_ele = html_child.xpath('//ul[@class="detail_comment_list"]/li')
                            for d in detail_ele:
                                dict_data = dict()
                                dict_data['supplier'] = supplier
                                dict_data['title'] = title
                                dict_data['pf'] = pf
                                dict_data['price'] = price
                                # 用户名称
                                dict_data['username'] = "".join(d.xpath('./div[@class="detail_comment_l"]/p[@class="user_id"]//text()'))
                                # 用户评分
                                dict_data['user_pf'] = "".join(d.xpath('./div[@class="detail_comment_l"]/div/strong//text()')).replace("分", "")
                                # 旅行类型
                                dict_data['travelTypeName'] = "".join(d.xpath('./div[@class="detail_comment_l"]/span[@class="user_type"]//text()'))
                                # 标签-分项
                                num_t1 = 0
                                for t1 in d.xpath('./p[@class="detail_comment_info"]/span'):
                                    dict_data['tag_child_' + str(num_t1)] = "".join(t1.xpath('./text()'))
                                    num_t1 += 1
                                dict_data['comments'] = "".join(d.xpath('./p[2]//text()'))  # 评论内容
                                dict_data['comments_time'] = "".join(d.xpath('./div[@class="detail_comment_tips"]/a/text()'))  # 评价时间
                                dict_data['comments_1'] = "".join(d.xpath('./div[@class="comment_feedback"]/p/text()'))  # 供应商回复
                                dict_data['time_1'] = "".join(d.xpath('./div[@class="comment_feedback"]/p/span/text()'))  # 供应商回复时间
                                dict_data['url'] = url
                                dict_data['country'] = country
                                if float(dict_data['user_pf']) <= 4.9:
                                    insertItem("携程旅游_台湾", dict_data)
                                    print(str(dict_data))
                                else:
                                    insertItem("携程旅游_台湾_1", dict_data)
                            try:
                                # elements = driver.find_element_by_xpath('//div/a[contains(text(),"下一页")]')
                                # time.sleep(2)
                                # elements.click()
                                # driver.find_element_by_xpath('//div/a[contains(text(),"下一页")]').click()
                                driver.find_element_by_xpath('//a[@class="c_down"]').click()
                            except Exception as ex:
                                print(ex)
                                break
                        driver.close()
                # 返回主窗口
                driver.switch_to.window(now_handle)
    # xpath = driver.find_elements_by_xpath('//div[@class="list_product_right"]')
    # for x in xpath:
    #     # title = "".join(x.xpath('./p/@title'))
    #     # pf = "".join(x.xpath('./div/div[1]/div[1]/div/p//text()'))
    #     # renshu = "".join(x.xpath('./div/div[1]/div[1]/div[2]//text()')[0])
    #     # pingjia = "".join(x.xpath('./div/div[1]/div[1]/div[2]//text()')[1])
    #     # price = "".join(x.xpath('./div/div[1]/div[2]/span//text()'))
    #     x.find_element_by_xpath('//p').click()
    #     time.sleep(1)
    #     driver.find_element_by_xpath('//a[contains(text(),"下一页")]').click()
    # 点击下一页
    # driver.find_element_by_xpath('//a[contains(text(),"下一页")]').click()
    # 点击下一页


def close_driver(driver):
    driver.quit()

def insertItem(tableName, data):
    my_set = db[tableName]
    my_set.insert_one(data)


if __name__ == '__main__':
    conn = MongoClient('127.0.0.1', 27017)
    db = conn["Test"]
    driver = init_driver()
    for i in range(1, 25):
        print(f"========第{str(i)}=============")
        url = f"https://vacations.ctrip.com/list/whole/r--120002.html?cityname=北京&p={i}&s=4"
        url = f"https://vacations.ctrip.com/list/whole/d-taiwan-100076.html?s=4&st=%E5%8F%B0%E6%B9%BE&sv=%E5%8F%B0%E6%B9%BE&p={i}"
        get_url(driver, url, "欧洲")

